{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NN back propagation（神经网络反向传播）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import matplotlib\n",
    "import scipy.optimize as opt\n",
    "from sklearn.metrics import classification_report#这个包是评价报告"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data(path, transpose=True):\n",
    "    data = sio.loadmat(path)\n",
    "    y = data.get('y')  # (5000,1)\n",
    "    y = y.reshape(y.shape[0])  # make it back to column vector\n",
    "\n",
    "    X = data.get('X')  # (5000,400)\n",
    "\n",
    "    if transpose:\n",
    "        # for this dataset, you need a transpose to get the orientation right\n",
    "        X = np.array([im.reshape((20, 20)).T for im in X])\n",
    "\n",
    "        # and I flat the image again to preserve the vector presentation\n",
    "        X = np.array([im.reshape(400) for im in X])\n",
    "\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, _ = load_data('ex4data1.mat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_100_image(X):\n",
    "    \"\"\" sample 100 image and show them\n",
    "    assume the image is square\n",
    "\n",
    "    X : (5000, 400)\n",
    "    \"\"\"\n",
    "    size = int(np.sqrt(X.shape[1]))\n",
    "\n",
    "    # sample 100 image, reshape, reorg it\n",
    "    sample_idx = np.random.choice(np.arange(X.shape[0]), 100)  # 100*400\n",
    "    sample_images = X[sample_idx, :]\n",
    "\n",
    "    fig, ax_array = plt.subplots(nrows=10, ncols=10, sharey=True, sharex=True, figsize=(8, 8))\n",
    "\n",
    "    for r in range(10):\n",
    "        for c in range(10):\n",
    "            ax_array[r, c].matshow(sample_images[10 * r + c].reshape((size, size)),\n",
    "                                   cmap=matplotlib.cm.binary)\n",
    "            plt.xticks(np.array([]))\n",
    "            plt.yticks(np.array([]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "window.mpl = {};\n",
       "\n",
       "\n",
       "mpl.get_websocket_type = function() {\n",
       "    if (typeof(WebSocket) !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert('Your browser does not have WebSocket support.' +\n",
       "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "              'Firefox 4 and 5 are also supported but you ' +\n",
       "              'have to enable WebSockets in about:config.');\n",
       "    };\n",
       "}\n",
       "\n",
       "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = (this.ws.binaryType != undefined);\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById(\"mpl-warnings\");\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent = (\n",
       "                \"This browser does not support binary websocket messages. \" +\n",
       "                    \"Performance may be slow.\");\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = $('<div/>');\n",
       "    this._root_extra_style(this.root)\n",
       "    this.root.attr('style', 'display: inline-block');\n",
       "\n",
       "    $(parent_element).append(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen =  function () {\n",
       "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
       "            fig.send_message(\"send_image_mode\", {});\n",
       "            if (mpl.ratio != 1) {\n",
       "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
       "            }\n",
       "            fig.send_message(\"refresh\", {});\n",
       "        }\n",
       "\n",
       "    this.imageObj.onload = function() {\n",
       "            if (fig.image_mode == 'full') {\n",
       "                // Full images could contain transparency (where diff images\n",
       "                // almost always do), so we need to clear the canvas so that\n",
       "                // there is no ghosting.\n",
       "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "            }\n",
       "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "        };\n",
       "\n",
       "    this.imageObj.onunload = function() {\n",
       "        this.ws.close();\n",
       "    }\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_header = function() {\n",
       "    var titlebar = $(\n",
       "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
       "        'ui-helper-clearfix\"/>');\n",
       "    var titletext = $(\n",
       "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
       "        'text-align: center; padding: 3px;\"/>');\n",
       "    titlebar.append(titletext)\n",
       "    this.root.append(titlebar);\n",
       "    this.header = titletext[0];\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
       "\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
       "\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = $('<div/>');\n",
       "\n",
       "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
       "\n",
       "    function canvas_keyboard_event(event) {\n",
       "        return fig.key_event(event, event['data']);\n",
       "    }\n",
       "\n",
       "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
       "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
       "    this.canvas_div = canvas_div\n",
       "    this._canvas_extra_style(canvas_div)\n",
       "    this.root.append(canvas_div);\n",
       "\n",
       "    var canvas = $('<canvas/>');\n",
       "    canvas.addClass('mpl-canvas');\n",
       "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
       "\n",
       "    this.canvas = canvas[0];\n",
       "    this.context = canvas[0].getContext(\"2d\");\n",
       "\n",
       "    var backingStore = this.context.backingStorePixelRatio ||\n",
       "\tthis.context.webkitBackingStorePixelRatio ||\n",
       "\tthis.context.mozBackingStorePixelRatio ||\n",
       "\tthis.context.msBackingStorePixelRatio ||\n",
       "\tthis.context.oBackingStorePixelRatio ||\n",
       "\tthis.context.backingStorePixelRatio || 1;\n",
       "\n",
       "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "\n",
       "    var rubberband = $('<canvas/>');\n",
       "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
       "\n",
       "    var pass_mouse_events = true;\n",
       "\n",
       "    canvas_div.resizable({\n",
       "        start: function(event, ui) {\n",
       "            pass_mouse_events = false;\n",
       "        },\n",
       "        resize: function(event, ui) {\n",
       "            fig.request_resize(ui.size.width, ui.size.height);\n",
       "        },\n",
       "        stop: function(event, ui) {\n",
       "            pass_mouse_events = true;\n",
       "            fig.request_resize(ui.size.width, ui.size.height);\n",
       "        },\n",
       "    });\n",
       "\n",
       "    function mouse_event_fn(event) {\n",
       "        if (pass_mouse_events)\n",
       "            return fig.mouse_event(event, event['data']);\n",
       "    }\n",
       "\n",
       "    rubberband.mousedown('button_press', mouse_event_fn);\n",
       "    rubberband.mouseup('button_release', mouse_event_fn);\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
       "\n",
       "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
       "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
       "\n",
       "    canvas_div.on(\"wheel\", function (event) {\n",
       "        event = event.originalEvent;\n",
       "        event['data'] = 'scroll'\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        mouse_event_fn(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.append(canvas);\n",
       "    canvas_div.append(rubberband);\n",
       "\n",
       "    this.rubberband = rubberband;\n",
       "    this.rubberband_canvas = rubberband[0];\n",
       "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
       "    this.rubberband_context.strokeStyle = \"#000000\";\n",
       "\n",
       "    this._resize_canvas = function(width, height) {\n",
       "        // Keep the size of the canvas, canvas container, and rubber band\n",
       "        // canvas in synch.\n",
       "        canvas_div.css('width', width)\n",
       "        canvas_div.css('height', height)\n",
       "\n",
       "        canvas.attr('width', width * mpl.ratio);\n",
       "        canvas.attr('height', height * mpl.ratio);\n",
       "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
       "\n",
       "        rubberband.attr('width', width);\n",
       "        rubberband.attr('height', height);\n",
       "    }\n",
       "\n",
       "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
       "    // upon first draw.\n",
       "    this._resize_canvas(600, 600);\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus () {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var nav_element = $('<div/>')\n",
       "    nav_element.attr('style', 'width: 100%');\n",
       "    this.root.append(nav_element);\n",
       "\n",
       "    // Define a callback function for later on.\n",
       "    function toolbar_event(event) {\n",
       "        return fig.toolbar_button_onclick(event['data']);\n",
       "    }\n",
       "    function toolbar_mouse_event(event) {\n",
       "        return fig.toolbar_button_onmouseover(event['data']);\n",
       "    }\n",
       "\n",
       "    for(var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            // put a spacer in here.\n",
       "            continue;\n",
       "        }\n",
       "        var button = $('<button/>');\n",
       "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
       "                        'ui-button-icon-only');\n",
       "        button.attr('role', 'button');\n",
       "        button.attr('aria-disabled', 'false');\n",
       "        button.click(method_name, toolbar_event);\n",
       "        button.mouseover(tooltip, toolbar_mouse_event);\n",
       "\n",
       "        var icon_img = $('<span/>');\n",
       "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
       "        icon_img.addClass(image);\n",
       "        icon_img.addClass('ui-corner-all');\n",
       "\n",
       "        var tooltip_span = $('<span/>');\n",
       "        tooltip_span.addClass('ui-button-text');\n",
       "        tooltip_span.html(tooltip);\n",
       "\n",
       "        button.append(icon_img);\n",
       "        button.append(tooltip_span);\n",
       "\n",
       "        nav_element.append(button);\n",
       "    }\n",
       "\n",
       "    var fmt_picker_span = $('<span/>');\n",
       "\n",
       "    var fmt_picker = $('<select/>');\n",
       "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
       "    fmt_picker_span.append(fmt_picker);\n",
       "    nav_element.append(fmt_picker_span);\n",
       "    this.format_dropdown = fmt_picker[0];\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = $(\n",
       "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
       "        fmt_picker.append(option)\n",
       "    }\n",
       "\n",
       "    // Add hover states to the ui-buttons\n",
       "    $( \".ui-button\" ).hover(\n",
       "        function() { $(this).addClass(\"ui-state-hover\");},\n",
       "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
       "    );\n",
       "\n",
       "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
       "    nav_element.append(status_bar);\n",
       "    this.message = status_bar[0];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.send_message = function(type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function() {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
       "    }\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1]);\n",
       "        fig.send_message(\"refresh\", {});\n",
       "    };\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
       "    var x0 = msg['x0'] / mpl.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
       "    var x1 = msg['x1'] / mpl.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0, 0, fig.canvas.width, fig.canvas.height);\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch(cursor)\n",
       "    {\n",
       "    case 0:\n",
       "        cursor = 'pointer';\n",
       "        break;\n",
       "    case 1:\n",
       "        cursor = 'default';\n",
       "        break;\n",
       "    case 2:\n",
       "        cursor = 'crosshair';\n",
       "        break;\n",
       "    case 3:\n",
       "        cursor = 'move';\n",
       "        break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function() {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message(\"ack\", {});\n",
       "}\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = \"image/png\";\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src);\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data);\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig[\"handle_\" + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "}\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function(e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e)\n",
       "        e = window.event;\n",
       "    if (e.target)\n",
       "        targ = e.target;\n",
       "    else if (e.srcElement)\n",
       "        targ = e.srcElement;\n",
       "    if (targ.nodeType == 3) // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "\n",
       "    // jQuery normalizes the pageX and pageY\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    // offset() returns the position of the element relative to the document\n",
       "    var x = e.pageX - $(targ).offset().left;\n",
       "    var y = e.pageY - $(targ).offset().top;\n",
       "\n",
       "    return {\"x\": x, \"y\": y};\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys (original) {\n",
       "  return Object.keys(original).reduce(function (obj, key) {\n",
       "    if (typeof original[key] !== 'object')\n",
       "        obj[key] = original[key]\n",
       "    return obj;\n",
       "  }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function(event, name) {\n",
       "    var canvas_pos = mpl.findpos(event)\n",
       "\n",
       "    if (name === 'button_press')\n",
       "    {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * mpl.ratio;\n",
       "    var y = canvas_pos.y * mpl.ratio;\n",
       "\n",
       "    this.send_message(name, {x: x, y: y, button: event.button,\n",
       "                             step: event.step,\n",
       "                             guiEvent: simpleKeys(event)});\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.key_event = function(event, name) {\n",
       "\n",
       "    // Prevent repeat events\n",
       "    if (name == 'key_press')\n",
       "    {\n",
       "        if (event.which === this._key)\n",
       "            return;\n",
       "        else\n",
       "            this._key = event.which;\n",
       "    }\n",
       "    if (name == 'key_release')\n",
       "        this._key = null;\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which != 17)\n",
       "        value += \"ctrl+\";\n",
       "    if (event.altKey && event.which != 18)\n",
       "        value += \"alt+\";\n",
       "    if (event.shiftKey && event.which != 16)\n",
       "        value += \"shift+\";\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, {key: value,\n",
       "                             guiEvent: simpleKeys(event)});\n",
       "    return false;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
       "    if (name == 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message(\"toolbar_button\", {name: name});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to  previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function() {\n",
       "        comm.close()\n",
       "    };\n",
       "    ws.send = function(m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function(msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overriden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data'])\n",
       "    });\n",
       "    return ws;\n",
       "}\n",
       "\n",
       "mpl.mpl_figure_comm = function(comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = $(\"#\" + id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm)\n",
       "\n",
       "    function ondownload(figure, format) {\n",
       "        window.open(figure.imageObj.src);\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy,\n",
       "                           ondownload,\n",
       "                           element.get(0));\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element.get(0);\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error(\"Failed to find cell for figure\", id, fig);\n",
       "        return;\n",
       "    }\n",
       "\n",
       "    var output_index = fig.cell_info[2]\n",
       "    var cell = fig.cell_info[0];\n",
       "\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
       "    var width = fig.canvas.width/mpl.ratio\n",
       "    fig.root.unbind('remove')\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable()\n",
       "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
       "    fig.close_ws(fig, msg);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.close_ws = function(fig, msg){\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width/mpl.ratio\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function() {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message(\"ack\", {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var nav_element = $('<div/>')\n",
       "    nav_element.attr('style', 'width: 100%');\n",
       "    this.root.append(nav_element);\n",
       "\n",
       "    // Define a callback function for later on.\n",
       "    function toolbar_event(event) {\n",
       "        return fig.toolbar_button_onclick(event['data']);\n",
       "    }\n",
       "    function toolbar_mouse_event(event) {\n",
       "        return fig.toolbar_button_onmouseover(event['data']);\n",
       "    }\n",
       "\n",
       "    for(var toolbar_ind in mpl.toolbar_items){\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) { continue; };\n",
       "\n",
       "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
       "        button.click(method_name, toolbar_event);\n",
       "        button.mouseover(tooltip, toolbar_mouse_event);\n",
       "        nav_element.append(button);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
       "    nav_element.append(status_bar);\n",
       "    this.message = status_bar[0];\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
       "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
       "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
       "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
       "    buttongrp.append(button);\n",
       "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
       "    titlebar.prepend(buttongrp);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function(el){\n",
       "    var fig = this\n",
       "    el.on(\"remove\", function(){\n",
       "\tfig.close_ws(fig, {});\n",
       "    });\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function(el){\n",
       "    // this is important to make the div 'focusable\n",
       "    el.attr('tabindex', 0)\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    }\n",
       "    else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager)\n",
       "        manager = IPython.keyboard_manager;\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which == 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "}\n",
       "\n",
       "\n",
       "mpl.find_output_cell = function(html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i=0; i<ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code'){\n",
       "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] == html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "}\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel != null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"799.9999880790713\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_100_image(X)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 代价函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 401)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_raw, y_raw = load_data('ex4data1.mat', transpose=False)\n",
    "X = np.insert(X_raw, 0, np.ones(X_raw.shape[0]), axis=1)#增加全部为1的一列\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, 10, 10, ...,  9,  9,  9], dtype=uint8)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def expand_y(y):\n",
    "#     \"\"\"expand 5000*1 into 5000*10\n",
    "#     where y=10 -> [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]: ndarray\n",
    "#     \"\"\"\n",
    "    res = []\n",
    "    for i in y:\n",
    "        y_array = np.zeros(10)\n",
    "        y_array[i - 1] = 1\n",
    "\n",
    "        res.append(y_array)\n",
    "\n",
    "    return np.array(res)\n",
    "# from sklearn.preprocessing import OneHotEncoder\n",
    "# encoder = OneHotEncoder(sparse=False)\n",
    "# y_onehot = encoder.fit_transform(y)\n",
    "# y_onehot.shape #这个函数与expand_y(y)一致"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = expand_y(y_raw)\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_weight(path):\n",
    "    data = sio.loadmat(path)\n",
    "    return data['Theta1'], data['Theta2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1, t2 = load_weight('ex4weights.mat')\n",
    "t1.shape, t2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def serialize(a, b):\n",
    "  \n",
    "    return np.concatenate((np.ravel(a), np.ravel(b)))\n",
    "# 序列化2矩阵\n",
    "# 在这个nn架构中，我们有theta1（25,401），theta2（10,26），它们的梯度是delta1，delta2  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10285,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta = serialize(t1, t2)  # 扁平化参数，25*401+10*26=10285\n",
    "theta.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feed forward（前向传播）\n",
    "> (400 + 1) -> (25 + 1) -> (10)\n",
    "\n",
    "<img style=\"float: left;\" src=\"../img/nn_model.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def feed_forward(theta, X):\n",
    "    \"\"\"apply to architecture 400+1 * 25+1 *10\n",
    "    X: 5000 * 401\n",
    "    \"\"\"\n",
    "\n",
    "    t1, t2 = deserialize(theta)  # t1: (25,401) t2: (10,26)\n",
    "    m = X.shape[0]\n",
    "    a1 = X  # 5000 * 401\n",
    "\n",
    "    z2 = a1 @ t1.T  # 5000 * 25\n",
    "    a2 = np.insert(sigmoid(z2), 0, np.ones(m), axis=1)  # 5000*26\n",
    "\n",
    "    z3 = a2 @ t2.T  # 5000 * 10\n",
    "    h = sigmoid(z3)  # 5000*10, this is h_theta(X)\n",
    "\n",
    "    return a1, z2, a2, z3, h  # you need all those for backprop\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deserialize(seq):\n",
    "#     \"\"\"into ndarray of (25, 401), (10, 26)\"\"\"\n",
    "    return seq[:25 * 401].reshape(25, 401), seq[25 * 401:].reshape(10, 26)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.12661530e-04,   1.74127856e-03,   2.52696959e-03, ...,\n",
       "          4.01468105e-04,   6.48072305e-03,   9.95734012e-01],\n",
       "       [  4.79026796e-04,   2.41495958e-03,   3.44755685e-03, ...,\n",
       "          2.39107046e-03,   1.97025086e-03,   9.95696931e-01],\n",
       "       [  8.85702310e-05,   3.24266731e-03,   2.55419797e-02, ...,\n",
       "          6.22892325e-02,   5.49803551e-03,   9.28008397e-01],\n",
       "       ..., \n",
       "       [  5.17641791e-02,   3.81715020e-03,   2.96297510e-02, ...,\n",
       "          2.15667361e-03,   6.49826950e-01,   2.42384687e-05],\n",
       "       [  8.30631310e-04,   6.22003774e-04,   3.14518512e-04, ...,\n",
       "          1.19366192e-02,   9.71410499e-01,   2.06173648e-04],\n",
       "       [  4.81465717e-05,   4.58821829e-04,   2.15146201e-05, ...,\n",
       "          5.73434571e-03,   6.96288990e-01,   8.18576980e-02]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_, _, _, _, h = feed_forward(theta, X)\n",
    "h # 5000*10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 代价函数\n",
    "<img style=\"float: left;\" src=\"../img/nn_cost.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "think about this, now we have $y$ and $h_{\\theta} \\in R^{5000 \\times 10}$  \n",
    "If you just ignore the m and k dimention, pairwisely this computation is trivial.  \n",
    "the eqation $= y*log(h_{\\theta}) - (1-y)*log(1-h_{\\theta})$  \n",
    "all you need to do after pairwise computation is sums this 2d array up and divided by m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cost(theta, X, y):\n",
    "#     \"\"\"calculate cost\n",
    "#     y: (m, k) ndarray\n",
    "#     \"\"\"\n",
    "    m = X.shape[0]  # get the data size m\n",
    "\n",
    "    _, _, _, _, h = feed_forward(theta, X)\n",
    "\n",
    "    # np.multiply is pairwise operation\n",
    "    pair_computation = -np.multiply(y, np.log(h)) - np.multiply((1 - y), np.log(1 - h))\n",
    "\n",
    "    return pair_computation.sum() / m\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.28762916516131892"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost(theta, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则化代价函数\n",
    "<img style=\"float: left;\" src=\"../img/nn_regcost.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the first column of t1 and t2 is intercept $\\theta$, just forget them when you do regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def regularized_cost(theta, X, y, l=1):\n",
    "    \"\"\"the first column of t1 and t2 is intercept theta, ignore them when you do regularization\"\"\"\n",
    "    t1, t2 = deserialize(theta)  # t1: (25,401) t2: (10,26)\n",
    "    m = X.shape[0]\n",
    "\n",
    "    reg_t1 = (l / (2 * m)) * np.power(t1[:, 1:], 2).sum()  # this is how you ignore first col\n",
    "    reg_t2 = (l / (2 * m)) * np.power(t2[:, 1:], 2).sum()\n",
    "\n",
    "    return cost(theta, X, y) + reg_t1 + reg_t2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.38376985909092365"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regularized_cost(theta, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 反向传播"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取数据和权重过程与前向传播相同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 401), (5000, 10))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape,y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1.shape, t2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10285,)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid_gradient(z):\n",
    "    \"\"\"\n",
    "    pairwise op is key for this to work on both vector and matrix\n",
    "    \"\"\"\n",
    "    return np.multiply(sigmoid(z), 1 - sigmoid(z))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid_gradient(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# theta gradient\n",
    "super hard to get this right... the dimension is so confusing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gradient(theta, X, y):\n",
    "    # initialize\n",
    "    t1, t2 = deserialize(theta)  # t1: (25,401) t2: (10,26)\n",
    "    m = X.shape[0]\n",
    "\n",
    "    delta1 = np.zeros(t1.shape)  # (25, 401)\n",
    "    delta2 = np.zeros(t2.shape)  # (10, 26)\n",
    "\n",
    "    a1, z2, a2, z3, h = feed_forward(theta, X)\n",
    "\n",
    "    for i in range(m):\n",
    "        a1i = a1[i, :]  # (1, 401)\n",
    "        z2i = z2[i, :]  # (1, 25)\n",
    "        a2i = a2[i, :]  # (1, 26)\n",
    "\n",
    "        hi = h[i, :]    # (1, 10)\n",
    "        yi = y[i, :]    # (1, 10)\n",
    "\n",
    "        d3i = hi - yi  # (1, 10)\n",
    "\n",
    "        z2i = np.insert(z2i, 0, np.ones(1))  # make it (1, 26) to compute d2i\n",
    "        d2i = np.multiply(t2.T @ d3i, sigmoid_gradient(z2i))  # (1, 26)\n",
    "\n",
    "        # careful with np vector transpose\n",
    "        delta2 += np.matrix(d3i).T @ np.matrix(a2i)  # (1, 10).T @ (1, 26) -> (10, 26)\n",
    "        delta1 += np.matrix(d2i[1:]).T @ np.matrix(a1i)  # (1, 25).T @ (1, 401) -> (25, 401)\n",
    "\n",
    "    delta1 = delta1 / m\n",
    "    delta2 = delta2 / m\n",
    "\n",
    "    return serialize(delta1, delta2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d1, d2 = deserialize(gradient(theta, X, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1.shape, d2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 梯度校验\n",
    "<img style=\"float: left;\" src=\"../img/gradient_checking.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gradient_checking(theta, X, y, epsilon, regularized=False):\n",
    "    def a_numeric_grad(plus, minus, regularized=False):\n",
    "        \"\"\"calculate a partial gradient with respect to 1 theta\"\"\"\n",
    "        if regularized:\n",
    "            return (regularized_cost(plus, X, y) - regularized_cost(minus, X, y)) / (epsilon * 2)\n",
    "        else:\n",
    "            return (cost(plus, X, y) - cost(minus, X, y)) / (epsilon * 2)\n",
    "\n",
    "    theta_matrix = expand_array(theta)  # expand to (10285, 10285)\n",
    "    epsilon_matrix = np.identity(len(theta)) * epsilon\n",
    "\n",
    "    plus_matrix = theta_matrix + epsilon_matrix\n",
    "    minus_matrix = theta_matrix - epsilon_matrix\n",
    "\n",
    "    # calculate numerical gradient with respect to all theta\n",
    "    numeric_grad = np.array([a_numeric_grad(plus_matrix[i], minus_matrix[i], regularized)\n",
    "                                    for i in range(len(theta))])\n",
    "\n",
    "    # analytical grad will depend on if you want it to be regularized or not\n",
    "    analytic_grad = regularized_gradient(theta, X, y) if regularized else gradient(theta, X, y)\n",
    "\n",
    "    # If you have a correct implementation, and assuming you used EPSILON = 0.0001\n",
    "    # the diff below should be less than 1e-9\n",
    "    # this is how original matlab code do gradient checking\n",
    "    diff = np.linalg.norm(numeric_grad - analytic_grad) / np.linalg.norm(numeric_grad + analytic_grad)\n",
    "\n",
    "    print('If your backpropagation implementation is correct,\\nthe relative difference will be smaller than 10e-9 (assume epsilon=0.0001).\\nRelative Difference: {}\\n'.format(diff))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def expand_array(arr):\n",
    "    \"\"\"replicate array into matrix\n",
    "    [1, 2, 3]\n",
    "\n",
    "    [[1, 2, 3],\n",
    "     [1, 2, 3],\n",
    "     [1, 2, 3]]\n",
    "    \"\"\"\n",
    "    # turn matrix back to ndarray\n",
    "    return np.array(np.matrix(np.ones(arr.shape[0])).T @ np.matrix(arr))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "If your backpropagation implementation is correct,\n",
      "the relative difference will be smaller than 10e-9 (assume epsilon=0.0001).\n",
      "Relative Difference: 2.1455623285988868e-09\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gradient_checking(theta, X, y, epsilon= 0.0001)#这个运行很慢，谨慎运行"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# regularized gradient\n",
    "Use normal gradient + regularized term"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img style=\"float: left;\" src=\"../img/nn_reg_grad.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def regularized_gradient(theta, X, y, l=1):\n",
    "    \"\"\"don't regularize theta of bias terms\"\"\"\n",
    "    m = X.shape[0]\n",
    "    delta1, delta2 = deserialize(gradient(theta, X, y))\n",
    "    t1, t2 = deserialize(theta)\n",
    "\n",
    "    t1[:, 0] = 0\n",
    "    reg_term_d1 = (l / m) * t1\n",
    "    delta1 = delta1 + reg_term_d1\n",
    "\n",
    "    t2[:, 0] = 0\n",
    "    reg_term_d2 = (l / m) * t2\n",
    "    delta2 = delta2 + reg_term_d2\n",
    "\n",
    "    return serialize(delta1, delta2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "If your backpropagation implementation is correct,\n",
      "the relative difference will be smaller than 10e-9 (assume epsilon=0.0001).\n",
      "Relative Difference: 3.1905824084956572e-09\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gradient_checking(theta, X, y, epsilon=0.0001, regularized=True)#这个运行很慢，谨慎运行"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ready to train the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " remember to randomly initlized the parameters to break symmetry\n",
    "\n",
    "take a look at the doc of this argument: `jac`\n",
    "\n",
    ">jac : bool or callable, optional\n",
    "Jacobian (gradient) of objective function. Only for CG, BFGS, Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg. **If jac is a Boolean and is True, fun is assumed to return the gradient along with the objective function.** If False, the gradient will be estimated numerically. jac can also be a callable returning the gradient of the objective. In this case, it must accept the same arguments as fun.\n",
    "\n",
    "it means if your `backprop` function return `(cost, grad)`, you could set `jac=True`  \n",
    "\n",
    "This is the implementation of http://nbviewer.jupyter.org/github/jdwittenauer/ipython-notebooks/blob/master/notebooks/ml/ML-Exercise4.ipynb\n",
    "\n",
    "but I choose to seperate them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_init(size):\n",
    "    return np.random.uniform(-0.12, 0.12, size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def nn_training(X, y):\n",
    "    \"\"\"regularized version\n",
    "    the architecture is hard coded here... won't generalize\n",
    "    \"\"\"\n",
    "    init_theta = random_init(10285)  # 25*401 + 10*26\n",
    "\n",
    "    res = opt.minimize(fun=regularized_cost,\n",
    "                       x0=init_theta,\n",
    "                       args=(X, y, 1),\n",
    "                       method='TNC',\n",
    "                       jac=regularized_gradient,\n",
    "                       options={'maxiter': 400})\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "     fun: 0.31677973007087407\n",
       "     jac: array([ -4.85498059e-05,   1.52435475e-09,  -2.42237461e-08, ...,\n",
       "         5.36244893e-05,   5.82601644e-05,   7.84943560e-05])\n",
       " message: 'Max. number of function evaluations reached'\n",
       "    nfev: 400\n",
       "     nit: 28\n",
       "  status: 3\n",
       " success: False\n",
       "       x: array([  0.00000000e+00,   7.62177373e-06,  -1.21118731e-04, ...,\n",
       "        -1.12048523e+00,  -1.00430109e+00,   1.42107038e+00])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = nn_training(X, y)#慢\n",
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 显示准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "       10, 10, 10], dtype=uint8)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_, y_answer = load_data('ex4data1.mat')\n",
    "y_answer[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_theta = res.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def show_accuracy(theta, X, y):\n",
    "    _, _, _, _, h = feed_forward(theta, X)\n",
    "\n",
    "    y_pred = np.argmax(h, axis=1) + 1\n",
    "\n",
    "    print(classification_report(y, y_pred))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 显示隐藏层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_hidden_layer(theta):\n",
    "    \"\"\"\n",
    "    theta: (10285, )\n",
    "    \"\"\"\n",
    "    final_theta1, _ = deserialize(theta)\n",
    "    hidden_layer = final_theta1[:, 1:]  # ger rid of bias term theta\n",
    "\n",
    "    fig, ax_array = plt.subplots(nrows=5, ncols=5, sharey=True, sharex=True, figsize=(5, 5))\n",
    "\n",
    "    for r in range(5):\n",
    "        for c in range(5):\n",
    "            ax_array[r, c].matshow(hidden_layer[5 * r + c].reshape((20, 20)),\n",
    "                                   cmap=matplotlib.cm.binary)\n",
    "            plt.xticks(np.array([]))\n",
    "            plt.yticks(np.array([]))\n",
    "\n",
    "\n",
    "# nn functions starts here ---------------------------\n",
    "# ps. all the y here is expanded version (5000,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASwAAAElCAYAAABect+9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXfUZVV5xp87mkSKCDgoAiND7733PvSidAgCWUCUBCKS\nCLJ0CRGJuBaCWWSl6KImNIEgvfcOQ2doAlKDoDExXc3c/EF+ez/nfJvvu/ec+w1z4H3+mW/OPvec\n3c5+n7fsd/f6/b4CgUCgC5jyflcgEAgEBkUsWIFAoDOIBSsQCHQGsWAFAoHOIBasQCDQGcSCFQgE\nOoNYsAKBQGcQC1YgEOgMYsEKBAKdwUeHuXn++efvL7zwwpNVl0nFpz71Kc2cOfPn/X5/kfHumzp1\nav/3fu/35lS1Ro7PfOYzA7dznnnmmVPVGjlef/31CdsoSfPOO29/wQUXnBNVGjkWW2yxgcZSkuab\nb77OtnPQOSsNuWAtvPDC+rM/+7OB72fbT2n7T6/Xq/w77DOH/d1RRx2lXq/3ykT3TZ8+XZ///OeH\nevYgmD179phrU6ZMTHD9d4Pcf8IJJwzczgMPPHDC5znqfT/euE42vvKVr0zYRklacMEFdcQRRwz9\n/EHm7GRvazvxxBMHGkvp3XZ+6UtfmtT6TBa+/vWvD9zOoRasYQFT+c///E9J0m9/+9tUxsfHvx/5\nyEfGfRb3MVlKCwCYUx8NqNdlkIWl9DtJ+t///V9JuT+8Xygb9j2jBu91hvbrX/9aUh5j/5jHEzL/\n8z//I0n63d/93XRtTo+fo74YOdtmvBgHr2dJANfntM//8ebv3IzSIk07mRfezlEjbFiBQKAziAUr\nEAh0BiNTCaGKv/M7v5OuYQSE9qM2SJkSL7DAApKk+eabL5WVVJ1PfepTkqT/+I//kCT98pe/TGW/\n+c1vJGWqXledRokSlf/kJz8pSfr4xz8uSXrzzTdTGeow7XMVY95555VUbe8vfvELSdLUqVMlVen1\nv/3bv0nKFJy+eK96jRq8g/7+zGc+k8pQhf7pn/5JUrXezANv5zvvvCNJ+ud//mdJuQ/9Pto52Sqi\n14s2Mle97F//9V8lZfXV5yxz3MFY80x/Fm3iWf5tzMmUT9TN1Xv+pr4+t+pmHmmsGYD5IY1+7IJh\nBQKBzqA1w2L1ZSWFGUh5hV5ooYUkSYsskr2WsJGf/OQnkqQnnngilX3sYx+TJK222mrp2r//+79X\nnuFM7q233pKUV3aXVn5fG9BO6sa/krTqqqtKkn7+859Lkp566qlU9sorr1TqtNFGG6WyXXfdVVKV\nXXAfLM2lMkzlv/7rvyRJ//3f/z2mfm0xnnSnf3/1q19JyoxPyqz2ySeflCQtvfTSqYyxdlYMk4S1\nOD772c9Kkuaff/4xv5tsMJfoBx9nWAX98IlPfCKV/cu//EvlXymPK+PmrJO2wWaclUzkgGoKH1vq\nwvfhjAnmCGt87bXXUtkqq6xSuUfK85B56eD5MDO/p4kmFAwrEAh0Bo0YVslljS7uZa+++qqkLGHd\nfoMkg0n8zd/8TSqDkR1//PHpGtIKqfbpT396zHt4lksoZ3WjAJLQJS9SA7borGjmzJmSskTbbLPN\nxvzuxRdfTNc++tF3h+Shhx6SJK2xxhqpbPr06ZKk5557TlI1FMAl9ChQsrvQZvp+8cUXT2XPPPNM\n5X7qKEnTpk0bU1+Y5GKLLSapKuF5BvPGmQlMexSgDm5/gvkttdRSkrKNVcqMEnuqsynm2csvv5yu\nwRSx08HApWx/hK26rccZ9yhQCjegH2F/Pn9gycxLHzfawDyVpPXXX19SZl9//dd/ncr4XtFCfA1w\nDWHgtgz9i0AgEHifEAtWIBDoDFqrhFBDqJ8bubfbbjtJ0oMPPihJ+ulPf5rKUN+gv1tttVUqQz14\n5JFH0rXVV1+98h6M2VI29EPHS4bPJnCajoEQI+nyyy+fyqDMqKJOdVFrUA9Q67zstttuS9foB1TB\nF154IZWxj3PFFVccU1d3NIwCtN1VQvqS0AVUIyn3wRJLLCFJeuONN1LZ22+/LanqjNh8880lZePt\ns88+m8p4PoZ7V8tmzZrVqj3SWMeCq3HMJfrY70UdQr3x8BXmtquXqMW0o6S2o3ZhhJfyvGgLvk3U\ne5+XqPWo+V63u+66S1I2zbj55aqrrpIkLbvssunakksuKSmrnHfccUcqY/vXmmuuKan63boZYFAE\nwwoEAp3B0Ayr3+9XVkZWaAzIns0BicS/999/fypjpT311FMlVaXVxRdfLKkqFZHSO+ywgyTp9ttv\nT2UY8mBhjz32WCprE0TqEol2YqR86aWXUtmiiy4qKbMcDOYOWIIzoVtvvVWS9Pzzz6drSFw2YHt/\n0n8EnLpr2Y2gw6K0wZrATg9OZQwwSLsBlTlBf3t9YB3OyHg+bNgzDTAXbrrpJknSJptsksqahKn0\ner1iqACMwI3KMAHe40yROQAjcxc9zN+v8TfzGKeMJC2zzDKSpN12201SdQ60gTNCWBtj4w4LGDyG\n8rPOOiuVMe9h1DBKKc89nAVSdqDRZ9/85jdT2cMPPywpt72tcygYViAQ6AyGFsu9Xq/CWpDIuEJx\nYUtZB8bus/baa6cybATYLtZaa61Uxv1u1/je974nKbuIPZgQuwlMxyV/m60BJfsN7/WwBqQaUvL1\n119PZdQFnd9tHEg8t0k98MADY9oAPFRAqoY8tAkc9d/COmCCbnNjXLjHbT+4/LG5wXb9dyuttFK6\nRlsIa/DQDiQ8fYBUl0YTvgETgCVssMEGqYwQGZgfjEvKDBPtgMBfqTofwNNPPy0p24TcFkQd+Jac\nmbXJx+YMC9ZHHzvLhNHed999kqqMmHCVFVZYofJ/KdtYmaeSdO2110rKc2a99dZLZbTzZz/7maSq\n7beJ3TUYViAQ6AxiwQoEAp1BI6O7u5lRGdj3h4tTkm644QZJ2ajphtWVV15ZUlYFnRKjXrqRfsaM\nGZKqIQuA+hCl7G7hJqoSv/G9Yqg1GBhRHaSs6nC/U2jc96gFvise1Qc3viQdeeSRknKf+V47/qbv\n2u7LKmXYgKajwnofoFKgGrkqi3pL/7jDYt1115VUjfRmjFCjMUJLWSXiPa4SNlV9XVVCreQ9PiaE\nJzCvcQ5IeZ7tu+++kqoqLkZl31nBt4Da5Yb1vffeW1JWL92R5d/XsHDVDocJY8g4SNm0guPGVfht\nt91WUlbt7rnnnlRGaIqHKDFXSqEwfAuEi/g4l77liRAMKxAIdAaNGJZLK1jTcsstJ6kqHfgbyVLa\nmY/h2HeEn3LKKZKq+ZYwWJLVwV36rPowD5dysK4mcMnOsw844ABJ0g9+8INUhtRA8v7lX/5lKoMl\n4nDwMAEcFHfeeeeY+zfeeGNJ1eBZ9mzVswZI7Xb4u1uf59BvODT8PliOOz5gGPwed7mUx8clKuPH\n+LtzhPGD3bjEbhLW0O/3i6Eb9X2DUjZQM3c9rASGguHYmTGMxZ0UsArGywNnqQ8s1Zksz2oCN/7D\naHFGlbKCoAF4Gd8iTgNny7ApD+uB1fF9l7JY8Ax3PMRewkAg8IHG0AzrIx/5SMV+g/RA2jjzwZWJ\nG9zD+ckLxQrMai7lQDUPAeA+3uMSAbuJb28ATbY5IIHdbQ+TxM3tOjzBcVtuuaUkVU4vefzxxyVJ\nF154oaTMRCVpww03lJTdx/43UsslJm3BHT+qvEm+i59ASVi0MwZCFvbff39J1XxY9AG2NLdX0faT\nTjopXWM8CRJ12wtzCHbj9h2X0IOiHjgKy0XCu6Svb8HyecbvsB36M6mXb0vBFsu8xA4r5XkMM/X+\n8lxiw8K/Gf6+/vrrJVXDYJjbbHXyfHTMM9pLeJKUs3I406UfsEf7ePEsxtLnWpMQlWBYgUCgM4gF\nKxAIdAZDqYS9Xk8f/ehHK2ofxkMovrv0oZLQR3erYmxk353TYNzAhx12WLpGIrwbb7xRUjWiHqDG\noIpKVZf1oKC+rqYQiX311VdLykZ4vx9D5le/+tVUxr4qDLDuSMCojCopZRWBMAHvTwzyGLu33377\nVNYksR2Gblcj6DscDq4uoR4S2eyGbMaaPZBO/b/+9a9LqhqTeTfzZuutt05lqJ7UwffzNd0z6Y4i\njMo8C9VNymouY3nOOeekMubcV77yFUnVOYuK544IVMHSnjyyd1Cv8847L5U1DWuYMmVKZbzoN/r4\nkksuSWU4DpjXl19+eSpD5ccMUkouuM4666RrqMh8a6igUnY2oTa6GaFJiEowrEAg0BkMJa76/b5+\n+9vfVgxuGOEIknPXNe76TTfdVFI1xICwAKSosyIkOPsHpbyK77777pKq+xLdvVyvQ+n4pYmA4dV/\ni7Gcay5JMWbiEvf8VkhhXOdeN4yTHlaAFMYJ4Tv82buG9PfA0SbtLJ2ijXEUZ4Fn0cCAWkr5C7M6\n//zzJVUZMGzLHStkYGDcneUQMEp2Bz+4wx0+g6Lf71fmLP2G+93DYGDwtNHHC2fHd77zHUnSscce\nm8owYnv90AroN38W9WFeeH/5fBgGs2fProSA4CBiDvoeV9oCU3KNAU2I0BR3XGGk9zrSrnvvvVdS\n1bjP+kAft01xHQwrEAh0BrFgBQKBzqCRBdPVDygoNNtjl/baay9JOUbl3HPPTWWoNRgmfS9cKYUF\nz8BY6An8iMNiD1TbaFoMoW6oJSkge648Pgr1EBXJY1owMnIqkFN2jPOeToS0K1Bnp970MQ4LT6Lm\nqZQHBe1zoyrjgsEVSi9l1YI6OfUnypm+L50M7BHlqIe8x/saVZC9ej43moxn/fkYvBkvj7X6kz/5\nE0nZweEqHiYNVHNX8W655RZJ2WEkZTULdc93LdTP63OVzPcvDgtXfVFr2WniZahvxALSJ1KeoziK\niLGTsppL9LyU5x7ODE+4WD+NPRL4BQKBDw2GZlizZ8+uuJZxZcIqXOpi1GTF9p3asDTS4Prqf+aZ\nZ0rKGQ78fqSv3887YWte5u71QVE69ZfnYOB3QySM48orr5RUZSywBAzJnrSM9jnrAhiafc8We9Fg\nBh4i0Sbq3X9LW3CGeER/fSeB77Uk1AVJ6lk7YKfOlOg/jLG+lxS3PtLYwxqa7g31dzOWsEAfE/ZO\nEmri7af/mYtnn312KoMV+R5KHEO8zxkZbJO+dwfGqM6YrJ/u7EZ35iUZGZwRk0GC9nr/E/3u+4L5\n9tlL6weFoH3ApH0vbZMQlWBYgUCgM2iUIrl0sjIrqKdAxR4DU3EmQegCerKHAlx66aWSpH322Sdd\nQwLhgvd9idQBnd0lVBOGBUq5hXDze+AcLnnq5szsiiuuqNTNgz2R7K7zI6mxq3iQLs/AluXhHCWW\nNij8t4wHbMAlL/ZJbGilvsWO6LYvQjPcTgJTxv7obA0mBwtxCd8kW0O9rrBI7G3OvrAdXnPNNZKq\nY0nICX1C9gwpz39nMYwT/et2HwKCYePOWDy4clh4iAr2sfqYet2438NnsFnxO7cJ87fb/RgTjvPz\nMBHGGVbqa0cwrEAg8IFGLFiBQKAzaBTW4PvzoLmoLn6yKzQfulxydfOv00j2F7qqxF41XKieJAxD\nLKqg0/g2qpKrH7wfKu2nvBAdjhHd348BmSj4UuoNdy4QyoGDwtUV6kBfu+rbpp0O6s64emgJBlTa\n4uoqIHLddy6gWvi5hPQD4++J8ugXVCMcFtJoUiRT77pTSMrOAkwbbgxnHqOaezgK6parc8xp+stT\nKtedJJ7CpanaWwdtxnTgKcoxafCN+q6JehJDd0pQNw+3oR8ZS09JhDqPScX7OvYSBgKBDzQaMSyX\nDkhDDGgudQmGw9jqJzIjgQm022mnnVLZgQceKKnKujB2824ProNhsWK7NB0V86BdJSbJ30gpZxIY\nVZFoXh8cB/4sANNxdlFvi0viUbWT58B4fAwYM+rr/UxdKHP3NUzEA45hXYynS1skOk4bf09TeKhJ\nPRuHh6jAlNhP52dBcj8MxFk+feMsBgN+KUlh6QAQ0OZcQgdjWc8a4deoo4ehEOgKW3SHBfX1MWGO\nEN7gzgWYWGl+BsMKBAIfaDRLLmRAsqDvIjmlrOsjYdx+Q/YDGAtSy5/hrtY6i/HVGelZ2moyKtS3\nhDjLxP4AK/GATqQqfeC2GtriO9hxd/N8t4c1OSl3WNCHvN9zPiFdsZ25vQZbG+zSs1kAH7O6TaMU\ndjAKZlVC/Ug0rytjwXz2cAvmL2Pq40b9/f766c6lkIM2J3YPCr4xn7O0mWvOiAFs0e1btN3ZcilH\nFmhy/Nx4CIYVCAQ6g1iwAoFAZ9BIJXSVq35GndN4DOP1vVv+DGizR/qS+M/VBH47nto3Garge8Gd\nC9Bq6LKHG/A3qo/3Qenct/oz54TKMB5cxUedLZ3wi+u8VO/SuKAq8AxXfScbzCHmlxu5UXWYu67S\n0A7Gy+c6Zd5Wnluas5Ol7pZQDyHxvxknN/5TT1cF678r7XLAkD+ZczYYViAQ6Ax6w6z0vV7vHUlj\nffDdwZL9fn+R8W74ALRR+nC0c8I2StHODmGwds5JahoIBAJtECphIBDoDGLBCgQCnUEsWIFAoDOI\nBSsQCHQGsWAFAoHOIBasQCDQGcSCFQgEOoNYsAKBQGcw1F7Cj3/8431P6NYlLLzwwpo5c+bPJ4qm\nnTp1at8Ts3UNCy200MDtnFN1mgz84he/mLCNkjTffPP1Pf1wl7DooosONJaSNM888/RJidM1TJs2\nbeB2DrVgLbLIIjr55JMHvp9NkKPKhtkGBxxwgHq93oRbF6ZPn56OkO8i9tlnn4Hbufrqqw/17MnM\nNzYszj777IG2oSy00EI66qijhn4+bRxvI2+pH0o7R5r213HHHTfQWErv5u/af//9x72nVLfSt8l9\ng7alft947S1tGD/jjDMGbmfrBH7jod6Q0uCP4iN4vxdEdq6TraG0K3+8yeKJ1cgOUE9KWHrW+71w\nlCZf6f+lPqC8nrWh9Kw5iXr2Ak8rTJK78bI1eBrkesLDUjK7UW6N6/f7xX4vZdegjGte7/rJ015v\n+qeUjJB57Mn9Rj2W7z/1CQQCgQERC1YgEOgMRq4SOp0kKR+JvUrnm5HL3ROIcXKJ007PlS1Vk979\n6le/qjyf90lzRl2s5+z2hHDkJ6fM+4ffefI6DMSc/eh57Xlu/cSTyQAqg9eXRHC815P7kf+7dPw4\nZ/v5WYU8n37x8cXpgToxWcn96nWQ8gk6tK100g3zzdU5cvH7fOPEI5Jblk45om2jUJ3eSy3nu+CM\nBSkn52O++TkEnJqDCuw57+krV/v4m/nsif/qp2r5eYZN1OFgWIFAoDMYGcNiNYcdSflUFaSJn7LC\n6SpIHU85zGpM2l0pr8as3i7JWMWRfKV6jQpIGDeUI4lgPi5h+Bt25HVEurnkQ1px1qKnVOZ8R1IV\nO/txiTcKUG+XgowH7BZWIUnbbrutpNzfL730Uir76U9/KqnaFpgF4+j3M5483+vgp443Be+krl4v\nzoqkP996660xv1tiiSUkVecnbMrPpCTNMgzHw2V4J0Z9n6den2FROpOT8XKmRD/WT/r2OjHH/Zm0\nwecefUR7/X7eXXJKNNF+gmEFAoHOoDXDYuWs22qkbAeAgXjieiQyrGvWrFmpjFNo/RTdO+64Q5L0\n4IMPSpI23HDDVMYpvUgGX7lHZecZz+ZSdwP72YNIGP6FOUnSyy+/LKnqNt5oo40kZen20EMPpTLY\nK3Y/r8OoGBbtgwm6VKa+nOjtdg/qwtl0/rs33nhDUtW2t/XWW0vK7MvDB2g74+hjWLdlNkH9lHI/\n+Zm/GSdnRSuttFKlPo888kgqg1kxTyXpvPPOkyTtvPPOkqQtt9wylXHeI3Xwb6N0+MMo4DbkZZZZ\nRlL+xvxcQuYj7VxxxRVTGf3v40t9GTe3L9OfzAFvZxPbZDCsQCDQGcSCFQgEOoNGKqGrfdC6Er1+\n7rnnJGXKiOomSc8++6ykrCasttpqqQxV4PXXX0/XVlhhBUnSqaeeKqlKYTfeeON3G/P/VP+1114r\n1rUNUPug/q6uoj5BpX1PF2oxdPmWW25JZa+88u5uhIMPPnjMNfrjpptuSmWoEby7dDZcW9BfqLVu\n5MbYzPvduErfo9q5Gke9fW489thjkvL4u8ufZ3Fuo4epuHF4UPR6veJZmnW3vyTdfvvtkqQnn3xS\nkrTccsulMvqbMXr88cdTGSE8vt1pzTXXlJTVIXc6oVavtdZakqSZM2cO3a4SSu3EOeD7gDErYDD3\nuvFN8/2uuuqqqYy57X3Gb3Ee+beJComa6KEwpRCYiRAMKxAIdAaNGFYpAI5V1d32rPas7G6cRhIT\nGIn0lvKqvPnmm6drMLhbb71VUjV84oknnpCUJYEbsd3QOwqUNnTTPiQZLm5J+vSnP135/a677pr+\nxmgLk5Cku+++u/IsnBNSDoNAmteN/U3hLBQWQZswiktZqtK/blwl2BDWccUVV4x5j9f31VdflZSd\nNQ888EAq22233SRJq6yyiqRq+EAT9Hq9yjwgbAB24e1nnh155JGSctCrJN1///2V+sDCpGxQd02B\neQwrmT59eipDY3j00UclZRYmNd9fWN9LSLuok4cO1cMZPLSG+vJN8n1J+Zv0djKWtM/XAL5X6uVO\nDGfcgyIYViAQ6AwaMSyXAEgrbB1u20HCwL7uu+++VMbKjl2rFIj2/PPPp2t77bWXJGm99daTlCWT\nJF1wwQWSsmTeYostUpkzhGFRCuaD3fgWA9gegXP+O8INqIdLbKT+t7/97XSNfsF24mwR2wl2EpdW\nbQJkfTzffPNNSdmm4i72tddeW1K2WRxxxBGpDMZw0UUXSaqGb2BjdDsJ8wQ7j7u4kfDYOHxLD0xu\nWPh4YV9DU3C7GHOO9rsdlX5iXh9zzDGp7M4775RUDS+BofCv2/VgJTAbDxZ1+9Aw6PV6lbGkzYQw\nYJ+T8vzCPuihCzBJ6kRdpawZEZYiZXsnY+59xrtpJ/NLymM/DIJhBQKBziAWrEAg0Bk0UglLUbnQ\nRty0UjY+Y6TcYIMNUhnqIUa8kovYwwOg6qiOrkJyHxTW9/m1iQB3dQz1BFrvLlmMm4cccogk6amn\nnkplGCDpA48QvuuuuyRJO+ywQ7rGXrZNNtmk0iYpq4SozK5+t8lK4WoE/YWK4GonbUeV9XlA+AUG\nWiLepTwPXCXkuagPuNClHBmOqv/MM8+ksmGzpAI3rDMGqDfs0ZRy2+jP66+/PpUxfx9++GFJ2UEi\nZXUIk4WUVU1UQsbW20QfeuiGhwUMg36/X5kHqJbsIsGJIWUVHlOFhwKhKmMU9znId+vfOaEKqH2E\nTEhZfX7xxRclVdvpfw+KYFiBQKAzaMSwnF3UswvcfPPNqQwpveOOO0qq7r3aaaedJGVJ6/vvkBLu\nBsaASOAmEkLKKzsSxaVFE2M0v3EGQZsxmrsko+4YNZ0tIH2otxuXMbYTROjPgk15yAMhILA2D+os\nZaoYFM5WaR8s1V3PMBOYg5dhTL3qqqskVYMNcUb4vKGdGLeXX375VEYfEVrh86DJeM6ePbsyXryr\nHgwsZWPyhRdeKCk7DKQcusCYuBH9wAMPlFTd6wizYXw9cJPfsvfQA1R9X+Ww8P6p5+Py96MRwfx9\nvNBmXnjhhTG/g315dg2exVz3b5kAcRils9kmmTeCYQUCgc6gdVgD+iqrrK+gMKxzzz1XUnVFxUX+\nt3/7t5X/S5kpebAf9hpsER5oh+SmXh7U5vaPQVEK3IPBkLHAwyVgAEgyt1WwLQXJ5FIZt+5tt92W\nrhFYyn1+RBUSHhuf92ebwwycSdK/++yzj6RqO8kcAQPCLiFle84aa6whqRoIiq3Cx5j+xP642Wab\npTLmErYf7wMPWh4GzhJgIdiinNFQL1iRl2ED+tGPfiSpGo5CUCbZRCTp2muvlZRDcjw4k/tgM86y\nPcBzWDjDQnuhHz27xtlnny0pfyuXX355KiPYk/npNmHq63Y2+ggNw9ky84d5DWuTmgUEB8MKBAKd\nQSxYgUCgM2idwA+1D+rntBe1hmhXjyhGDdljjz0kVVU8KCU7+qVMZ1Eh3HiNuojB0yNtS2fBTYRS\nhgdoL+qJU3hUHvbWkYRPyml+ocJOr8855xxJVTfwjTfeKClTe1cPcDRg1Ec1bAra6aplPSOBq4uE\nKmCk5h4pG7AZf98ziUHeo+aJWKdf3IjLfGEfpjsxmoynVHVK8Az6050O1Of73/++pOq+UNT7b33r\nW5V7peyAIOJdyo4lVGEfe1TUvffeW1JVpcdJMSzqDglMJaifrhKywwQHgo/N0UcfLSm33Z/LXPFx\nwPHGfHKzh6fRlqpOiSbzNxhWIBDoDFob3VklMZR7YCQMiRXaDbgY5ynz1RYXq+9huvrqqyVlw56v\n1IQ8YPB1SeZ7yAYFLnCXIkhomMFSSy2VypAoMCzfL8X7kXaeDwjpU0obC5N0BwJ1oK/aHn9FO73v\nGUfa5EZn9kxutdVWkqp7PTGGY0h1No3kdWMsgaZIdj/O7HOf+5ykLLmdfTcNa/D5Qn0Icp0xY0Yq\nw3lA/Q477LBURpAzxnQPqiUchDkojTXSe3gGbcJB479zA/gw6Pf7xQMg0Aa8jPoSguDjxT5d2P7u\nu++eypiP/iy+N+a2Z1KpB1o7m28S1B0MKxAIdAaxYAUCgc6gkUrotJz9YRgu3ciGmkB6EN8HBuVG\nhXDaieHSVTtisnbZZZfKe6Uc2Y466vVrsl8J+PvraqIbRj1ljFRNhwK9RgX2hH51dVHK8UuoiR6L\nVj/fz/daejK9QUE/eRQ4BnXa50Z/1Bbud0rPWLOXzssYl2uuuSZdQxWpx7BJ2ZjN2HnsVdMTZdyR\nwrvZD+dGYvZpcm3TTTdNZajCpF/xPaOoVO6MYSxRg7wvMXpzzU+NGlVab+YJfevfDO2kHq6qMVdx\nHnkaGNRcTxXF/OeZHm/HGNI+7wNXqQdFMKxAINAZNGJYboyGMRDN++Mf/3hMGUY4N6KT+hdJ7hL5\n3nvvlVSVVhj7KPOVmv1eGFZdYrY5l9CZD2wIqeX7FWEJGFJ9Fz/PgM2U2J+73DFmUuZudd5Tj/qX\nmu3w5106uBg4AAAgAElEQVTO4qgfY0fmBClLVRikM0ukJXPDdxiUDsuA3cAafV8izBGJ7W1rdHDB\nlCmV0AX6m/AXZ3fUmxAGD5GBWRElfuihh6Yy5oX3CbsiGEMfZ+Y2c8br0DSBn1RmkoQzeHYPot+Z\n175DBWdOPSW3lMfXxwvnSz1hoZSZFaEqzvLiEIpAIPCBRuvAUZgRK6evxkgRgglLxwUhdVwybb/9\n9pKqgXnkjkJauf0GqcLK7mVtpJWzIaQVzMcZIW1AgnmAHpKIU4PdvgVLczcwbMSlVB0lJtl0j10d\nBLFy+rYHAiMtCUHwPZOwCdiQS2CYHIxJyvsSkbLOpmkL9ioPSWgS1tDv9yvzAFbBc91+dMMNN0jK\nbNYPYGB8CaB15lg/cELK+yNph88Zxp7wGB+/pgen9Hq9CsPCLkU9nGERssDc83x0vB8txvNh8U36\nd472Q394WA/vxMbqrKrJnA2GFQgEOoNYsAKBQGfQSCV0NziGdOidR3JDHzGsozZIWT3AsOfRzNBI\nV0f4G7e7qwaoHBgE3cDadO9Z/R0AY7RH0NfPqfMTY2gfZe7KRR3w+taj9t25gAEYGu8qSZszCr0t\nJJJDtXCjM9SfcfTQjvpOB3eTowa7usR9OEy8nTyfeeb1azqernYD1HwPX0EVJL2OJ9ajT6iPn9zE\nDgg/q5C5QrtJ2yJllR9TiodrNNmdAXzO0lcYxT3pJfVlrnofMK+ok9cHtdbDl1AhmQ8eYkMdMF/4\ne5qkRAqGFQgEOoPWRnckJdLQE6XBPAiw86BJ/sZA7fsM66cPS2NZlxsmPcGb/15qx7AcSFcM7N4W\npDLsyaUlBkgkmdcHA76ftQfT4D7PYoBBG6nsEqpNAj9nzEhT6ussF6nKYQyk95UyQ6be/jvGx7NS\n0EclAzMGXcbRDclN2tnr9SpzAgcPdXTnBUyM4GXPSEH9cTa4cwX2VDolmd/52X8wIb4fZx6j0gro\nq1KiPJwEjLczUMpgU6XwAw814T3MGQ9dwPnCM9uOZTCsQCDQGbRmWPV0ru6CRnJhp3BXPVkAuMel\nbwnYyJzZgPG2MrQ5/sqBNECCuW0HGwVMwpPw8zskqDMKJLzbCGAepO8tbQ9qc8rzRKgzQQ/D4BqS\n18caBkx93Y7BMz3YFlaJZC8Fz44S3mf1vE1ubyMEgOBJD0XARlNnJ1JuL1tdHKVDQ8Co5ud4oD/d\n5sh7+a6c1cH6uOZlPMvHl/spc0ZWb1/b9gbDCgQCnUEsWIFAoDNorRLW4QZn1EWMtK4OQcuhpBOp\nAVDJUe1kHxbQYoyNrirUo/w9xIB2oUY4JS7R4yb7qyYDqHau8jO2qBZef1RC2uu/w/DqY1dv+2So\nge8F3o3Tw8MtiPCnPl5n/qbMo+dxvLiaOJmq+3thvH4shb6g2pXqOuiY0C98355YkmeManyDYQUC\ngc6gN8zK1+v13pH0yoQ3zr1Yst/vLzLeDR+ANkofjnZO2EYp2tkhDNbOOUnFA4FAoA1CJQwEAp1B\nLFiBQKAziAUrEAh0BrFgBQKBziAWrEAg0BnEghUIBDqDWLACgUBnMNQ+kHnmmafv2xi6hGnTpmnm\nzJk/nyg4berUqf2m22PGi2mbU9s0llhiiUlv59yAn/3sZxO2UZLmnXfevh++0CUstthiA42lJM0/\n//x9MqB0DYssssjA7Rxqxi6wwAI68MADh65QPTXL+4Hvfe976vV6E0YCT58+XVtssUWjd8wNC9Zp\np502cDu32WabSavHRAHJbfvju9/97kBR3QsuuKAOP/zwgZ9bP+F72Hp6u+vpgIbdB3vSSScNNJbS\nu+majj/++IGfPTd8k+BLX/rSwO2cVBHLxmY2WHomQjYDI+V9Myyd6IPPxk025c6JPEKDgrqQ1dI3\nRlNGvT2/lf8N3q/N3XUMsgPC76Etpd+VDpFlIzXt9dxa9WdM9kc1XlsH3QnCXPejq8h3xtz1zLj1\nw2XnhoUDDNLm96u+c89XHwgEAhMgFqxAINAZjFwldKoI7UVd8IMJSJtLamQ3AKM6+unB0Gly7bhK\niAoGlfUE+aM6hALwXs/3hCPC0yYDDjygnd4m6u1qBM9FZfLDAci1NOrcYCUVgDHz3Ea8n7Fy1Re1\nn7TDflpwybjPXKDtnj64rl5O1gb9Utpt2uupoQFtYl67WkcbXe2jHeQP81xZzBlUYT8oYrLMAt6P\n1LeU8pjDVCjzPFqMuY8vecFIBe1H2dGftIkcZFIztTIYViAQ6AxGzrA8CyOrNm5lZxIcSLHoootK\nkh577LFUxsrrh1hy5BcHPHCAq5SlOhkP/bCDNgeMjgc3mCMdeZfXDYmE5OUgWCn3gbNAXNOwRD+U\nFWbFv34wbRsnROloKK75mAH62aUyx5lx4IizlnvvvVeS9JOf/CRdg2nSjy6xmRMwbR/DUYRiIO1h\nAp/97GdT2axZsyRlBuhHzVFH6uPHg9FPJXbBcV/33HNPKlt11VUl5XF7++23U1ndIN8W1Nf7kffS\nn/6twfpghv490XfuQOPA5LpjTMrzgP5ZeumlU1lpbk2EYFiBQKAzGBnDQpr4gagcR7/llltKyked\nS5lBcI8zCXT9Z555Jl3jKKEddthBUlXnZ4XHTuR2H7fBtEHdPuasCMmLHWb55ZdPZUhoJJLbOJBq\n3k4kHod0eo78OltzhtWmTaXjzevhGFJmHYwxh6dK2fYGO9pll11SGQfpYs+TpGWWWUZSluKMoZTt\nO0hzt0nWj5VrA9isB1zCFLnmNkcOjq2Pg5TngGsYzMd9991XUp7r0liWxnkAUjXEow3qNihnhIwv\n73K2TL8z97w+zAHGVMptXnPNNSVVj2yDLXIIb9twiGBYgUCgM4gFKxAIdAYjUwlxV6ISSFm9wLjs\ndP7++++XlOn1brvtNuZ3jz76aLo2c+ZMSdJKK60kqar2oaKgYo0qCtcN2VBm6jZ16tRUhqH2oYce\nkiTNmDEjlaE2YWR1Ay/qnxtZOWYKFcFVbFzt9IEbPt1dPChK/YQagPrgY3bbbbdV2uKqL8ZV1NSr\nr746ldF3O+20U7r2/PPPS8qGeFexn3jiicq7fRxGoS7hNKDOrtpiCMYIftNNN6Uy1EPURd9XWz/q\nSpLuvPNOSdmc4XP86aeflpRVf1fJ/BnDorT7gPr6PFtrrbUq73KTzNprry0pq7APPPBAKuNZX/jC\nF9K1E044QVLug0MPPTSVPfvss5Kyaca/Gw9zGhTBsAKBQGfQmmEhpZHMK6+8cipDeiClWLmlsYGJ\nLuWQ3Ntuu226xmoP48BYK0nrrruupLz6uzG6TdChS33aiXEVhiBlAyRGd2clSBgOk91www1TGUZb\nZxB1tlY6sBWJjSFTaiatgAc08j4YrEt7DOWwXSSr16XkXIDJuNMFp8mOO+44pg6wG8IAPKjUwyUG\nRa/Xq/QxzAgDu4cUML+Yn+6GZ1ypjxuxYSg4hfy3hDo8/PDDqeyWW26RlPvUA1W9L4aFB53CFgml\n8foSeoNB3vuHa4QaoQ1J2ZniThU0IeYFzgkpO53ou9LBtMMgGFYgEOgMWjMsVnEYCBJDygwFiYZt\nQsrMh5CHSy+9NJXhJi2xI5jW6quvnq7VAxldWowqrIFn8g4PKNx1110rdYMlSdI666wjKUtsl6RI\nMrctIIUffPBBSdKxxx6byggLoA5uzyltCxoUbsuiThy97mPw5JNPSsrj6q5t+of+9kBQ/nbJS39g\n0/Awla222kpSZi2wVH/+sPB20F6YDOPn70Jj8DlLnWG4zn498BJMmzat8r5LLrkklcEU+Z2HQ7Rh\nWN5O2C7z8vHHH09lSy21lKT8/TmTZJxh+V435qAHAfMtYL/2foHFom05q2oSBBwMKxAIdAaxYAUC\ngc6gtUqIWkKUq6uEGE1Rg9xFv8cee0iSNtpoI0lVo/Gyyy4rKe9Bk7LqQPTwNddck8oIm8Co6CoO\nLvi2gBajfroKg6qA42C99dZLZeyjpI64syXpzDPPlFSNdMdYTXT3I488ksroR/ZTuivcVbBhUco6\ngFPDDdLUnTFzBwtqDGqEhzwQokHIhpRNCaiSHlHO3xjpPQq+iaG23+9XVCVUluuuu06SdN9996Uy\nVCXa88orOREmajfqzde+9rUx9SplsCDkxJ0xmAqY16hhbeH9Q11oC+EwUv5W6vsApTy3caQw3lLu\ng1tvvTVdu+KKKyRlNde/P97J832cff4OimBYgUCgM2jNsJD2GGlvuOGGVHbyySdLko466ihJVRc5\n0gYjs++lgkm48ZyVmfvXX3/9VIZxkbAJpLyUDaTDgPe6tCRkAqnggXZIS6QIYRZSZkxI5dtvvz2V\neYYKgNGX8AdncvV00u7iL+VvmgiwDg8bwAWPcdudGxhvYUruwsdJwPg4M8FwTbCiv4c2eF/wbhjQ\nc889l8qc1Q0Dn0vMPcbS6wVLZrydFRL4yl45ZxIwSt/7CdthDN2YDutCM3HG0jTwudfrVRgW78Xw\n7YG7GOD5btFcpPzN0E8eIEvg86abbpquoR0R3uChOzAs5sdLL72UynxuDYpgWIFAoDNozbBgIazK\nvoLi5oR13X333akMyYKkcbc8kujaa69N15AEnAziNi9sXaVMCk1cxCU7CbYNbDo777xzKsNud8EF\nF0iqBujRL9TRbRUwQrevYM/imh9RVT/oAHYrNQuoLGVrKGU6BbBKJKn3M20nePDggw9OZTBOz1bA\n+CN5PUwE1zlMy+2ipXoNAp8HsDue5e5+QhHOO+88SVW7zxFHHCEps2Zvz8033yxJ+ou/+It0jfn/\nzW9+U5K03377pTK0AFiQP4s6DIt+v1+cuzAltyHDOPlencUSanHMMcdIki6++OJUxnx0bYnwFpin\nb+XBvssz/XdNxjIYViAQ6AxiwQoEAp1BI5XQVQjUH6ifG3Ch0yWaCu2H/vreQJ7lxrvttttOUlZD\nfF8WYRAYbt1N3yZ1sCevw3BZyhaAgRm66yEXXKP+nq2BaGGeLWXjLXu9MDxLWUXCSOxqcZMd/vSl\nJ9arp3t2IzJq+0EHHSSpGrkNvUd192eyl80T8fEsIqx9XyQmBNRE77Om2RpcfcXAfMghh0iqOixw\nEBx55JGSqnP9W9/6lqSsyvvcQuX3cdhzzz0l5fnh82mzzTaTlEMCXPVvGrrxm9/8pjJeqGE4Kjx8\nBnMCY7LCCiukMr5bnBKlcAzPYsH3hxEdNV/K3yIquX+bsZcwEAh8oNGIYZV2hOPmdKMyBliC09wQ\ni9Tk0IILL7wwlfEMN1JiBMVIT+CplIMyMVz+6Ec/SmUuuYaFS1fei4vX97chnWCLLsk22WQTSdnA\n7e572uk732FRSEU3xsIqYbWeEaFJEB4S3w8DgH3ACvy5hx12mKQsXT0ol/14sD7Ph8X4ePgADAz2\n5UZf6gDzKaXPbgPmI3XwYEbeSR87oyMQGrbmB04QfOrOj/qJ4LAqSbryyislZYbj4TdNjqbr9Xpj\nDnXAoE44jNcXJggz9j6G/ZL1xJ0eOAvINiGNzbvlfYBTDmeVrx1N2HIwrEAg0BnEghUIBDqD1kZ3\ncNFFF0mqpp1AnUBN8P1uxOEQWbziiiumMox3rkJAq3mmpweGxqNKeIxUm3PsvJ0YxqHS7hDAUE0c\niicqhF5jQHYaDA13NQcDJs/y1B71RIVuYG0S0U/7XCWqn27iz4XeQ/l9rDHMco9HpONI8Pg01GYM\ntJgGpKyi3njjjZKqxuomZ9lJ1Vg3xpLYOt9Hd9ppp0mSzj33XEnVtCvf+MY3JEmnn366pKzqSln1\n97Te7Grgma7qYfQmlY6rWG0cRW4gxwGGqcJ3oWDCQSV38wL9gsnCdy2gQpbawrx0xxvXiMH0GDNP\n6DgogmEFAoHOoHWkOwwApuRuXf5GijgwfLLHzneEIyXcYI7BkGhd38GPBOAel3xtUgeXQJucuSH1\nKfMQDcqot7vv2afmjgokGBKvtLsdA6YzM2chw8KZb/3gEA9PwJBOmbNFjM4khPMofCS7G52ZL4yZ\n73SAeTKu7nJvk/IaEDLC8z1sAocN7yRMQ8rZKsjG4eEQRPr7nljGF1bh7AuWSpS4M8cm7n7gz4E5\nogH4vsj6WZ4+B2gD13wnAHPb5x5hEDii3LjPfGc++Hx2DWFQBMMKBAKdwchsWEgfT2ELCyHnj7MA\nnsE9fvournyX7vVjtjyTApIeu48HKDZxEZdAPXmeSx2kDYF6LjmQSPzrjA+m4lIHxoqUdYmNDagU\nJNqEeTAGLtEJL6BObo/DvlY6rbu+L3KNNdZIZYzVa6+9lq7V95a5DYdr1MHH0AMPh4H3D8+DefjR\nU7wbpnLZZZelMuYj7MhDPrbYYgtJVTsdfQKT84NLCFHhmc5Y2rDIEtPGDulH8GEne/HFFyVV+5U2\nEMLjrA0m7KFDXCOo1Nk1LBR253O9iVYQDCsQCHQGsWAFAoHOoJFK6JQVFQY1yMtQIUqnEmMQhBKX\n3NWu+qDu8a+rCXUVyevQxkVceibqk6fGwHBJX3iqFNqFmlo6/82Nt9B3jO4eNYwzgr52d3wbQ60D\n1YB3eJQzqg0OBFcx+Jv64i6XpL322ktS1bnAHjpUZFeveAbt9fnTVF0qRVgzbu4IYT5yv++oQG0j\nfYrvWmDMXR1iXhLe4GEgzAfa6A6MNqE4pbmPase3I2UTDuEY7lxgDmKYL5l53IBPGAMONFexcXBQ\nFzcnNVHvg2EFAoHOoHVYA5IYV6av4hjjkJDOipDcGF9Lybxc0tQlv0tk/kYClk5sbgueScCqS0TY\nEPe4YR3XMK5tbxN18zAInk9bSnsEmxqe3wsulZGWXHNWAPtgXJ1hYcCGWXnYCfAARBgMRlh3hSOh\nkewlg/mw8GcQLsPzSb4nZfa4yiqrSKqm8YWJ4WzwswhLc4/7mCsEZErjn6XZRivw+U6bYXPuqOKb\nZHz9jMl6YLJrDMADvWFWzB1nZDwL7aNUv2EQDCsQCHQGrRlW3XbCaitlSVY/sVjKLARXrz+nHvIg\njdWBPfwfjCqEoYS6NCidhkudSvm4YFreTspKQXhg1GxqItSPavK6YY+DKXh/w6zJJ+UHF1Dmbmyk\nNtK4xCp4funU5jZgXvGvbxFhKw7swu1uMERYpNu+YC8l5l/fOiblvii59kcRHCvluca/bieG+ZTS\ninONdrqtlCBb7NPSWA3Dt83Vv8m2bQuGFQgEOoNYsAKBQGfQWiWE0uKedYqLYQ8Do9P+OjUsqQR+\nD7R0VGEKwwJajUriVBf1FuNqKcSA300U3Tte+0YVujAe6iqXR+2jQmEo93AS1Ie6oVfKqog/e7zk\nbaNSid4L1IM6uJEY9QYVfry6+HigOpcSDDJ3/USpNns/BwXvKDm9aBdlbn4Z79tErfV7aDPXPNof\njMr5FQwrEAh0Br1hpFmv13tH0isT3jj3Ysl+vz/WR2v4ALRR+nC0c8I2StHODmGwdk42/Q4EAoFR\nIVTCQCDQGcSCFQgEOoNYsAKBQGcQC1YgEOgMYsEKBAKdQSxYgUCgM4gFKxAIdAZDbc2Zd955+370\nVpew2GKLaebMmT+fKDht6tSp/aaHdbL94P2MbVt88cUHbud422Pmdrz22msTtlGS5plnnr7n2hoU\njGGbLSX1eTDss5ZYYomBxlKSPvaxj/X9gIfJxigzaCy55JIDt3OoBWvBBRfUEUccMXSFSoNfb2Tp\nIx+vI4ZdFE488UT1er0JI4GnT5+uHXbYYahng7lhwTr55JMHbudBBx004fNKY1bfV+kYtO2DjP94\n+PKXvzxQVPcnPvEJHXjggUM92+sz7Mfo++7Yu0d/Dbt/8LTTThtoLKV393d6OufJRilNUlP84Ac/\nGLidrTc/11Ga4OQF8lxBbJwt5QVispRyQXGfdxLvmRsWjPfCeHXzPqvnD3q/NnuD8SSpZ7Csj8tE\nHzrtrI/dRBjVJlpHqY/5IEvjRVtL+cl9jpP3nNxf5AorvXNObGz3d9D/aBP+/dXbPtHiVD+azjeA\ncz9lbedz2LACgUBnEAtWIBDoDFqrhPX8UH5SMalhyTfEqcJSpo2cFuvpY3mG02vyFAGnsORsIum/\nHzE1ahViEOpeOmgCej2oulpK91w//mky1Ii6iubvJAV0/RRuKedDK6nzpXryXEwDnnIalA7iGIW6\nj1qCSusHoNTnnh8cUVd9SuqNH0BCTjAOq3j77bdTGW3jWX401qjGlefU+9rLGCfPCcb3VLfBSVmF\n9GO+OFDmmWeekVTN+8Wc4V/Pr9ZEPQyGFQgEOoORGd1Zqf3IKiQXyfpdiu64446SpOWXX15SNXE9\n7Omtt95K15AAHHLhKzWSDEbnh1k2DVF4L1C30gEZvMvZBVKqxJhKhl2YDUdolQ564H0lj1QTOAut\nS17vP8aPMs8qCrMgU6cfzMkRZ15f2DBhMs7M60ebjaqdgDFkfvpBE1zjMBA/4gqG8uabb0qqHoTC\nGPrxZpQzt2fMmJHKbrvtNkm5v+kPKfdJW9BOxsu1Eo4zY045++Lv9dZbb0x9yJ665pprpmu0AS3I\nvz/eXcrG64dbDIpgWIFAoDNozbCQLOjgns8ZtsU9G2ywQSrbcsstJWXJcvfdd6cyWNfmm2+ernG8\n0IsvvigpH/8tZVsEksBZwahyZyMZkD4udZAs3MOhotLYeB5nD9gx3H5An2ETgrFI0ssvvywpt2+F\nFVZIZaU82k2ARITxOKOp234efPDBVMbx7RxAusYaa6Qy2BdHiEnZzsgzOdLcnw+7gTlL1WPrm4Kx\ngPnANqSxDO7RRx9Nf++8886S8pj6UWawyHXXXTddY27fcsstkqp2n6233lqS9Oyzz0qqHlg7ijZK\nuZ0cQ+dhKMxZWI5/J8yrDTfcUFKVLTPOl112WbqGJsQ3wdF9Umbhm2yyiaRq29zeNyiCYQUCgc4g\nFqxAINAZNFIJ3UiMYRHDulM+1Ir77rtPUlX1gaaybeK4445LZTfccIOkqtoFFT3ttNMkSeuvv34q\nc2ouVdUoVKu2qB9nhitXygZyKK47BKDc9WOVpHf3ikllqv70009Xfu/3oW65obSNSujjiTMB9W3F\nFVdMZVB/xvH1119PZWxnom6+r41nuNNl5syZkrK65yEvq666qqQ8jq461Md6ULjhnrFE/fYy6orq\n/ud//uepjPl49tlnS6qqkjzTwxMI2SkdpUVf0jZXRevhK03BfKHfXbXm++FfxkOS9txzT0l5LK++\n+upUdtJJJ0mqhnvwN/PB649DY5dddpFUPd7NHQ2DIhhWIBDoDBot5S7ZMYwifd54441UhgSizF2h\nbNREWp9++umpDAm7/fbbp2tIqZVXXllSNtpL0rXXXispS3VnZm3CGtwFyzMxnGJclrJ05X7qKGWp\nhoHXGR+Sz1kUbcCl7EGwF198saRsGHepPKqsAoztJz/5SUlVRoPL/6yzzpIkXXfddansD/7gDypt\nccZHPR9++OF0DcaMlCVUQMpsndACD+wsBZhOhClTplSkPmOBtN9uu+1S2TbbbCMpM11nTDh6KMMJ\nIuWxd4PzU089JUn68pe/LKkaOHrKKadIyobtq666KpW1GUufs/RVySEGgzznnHMq90jZscVcRxOQ\n8vz1eYlBHceZj+XnP/95SdmR5qy8CYJhBQKBzqARw/JVHOlMAJrbj7DzIN1wC0vS0ksvLUl66KGH\nJFVXf1iRS3BsXqzQrgsjMXm3B5y65GgD6kRbPNAVexO6/BZbbJHK0P+p97333pvKuN/tWpdffrmk\nzGy8DOZBm9wm6G7xYVHaRsM1ZxGwRdrnDA+Wuemmm0rKfSLlMA/GWpIeeeQRSZlRuuscm9Jjjz02\npg6rr776kK17F86wfvnLX0rKfesaw4033igpsyhn6MxxmMcJJ5yQyv7xH/9RkrTrrrumazDixx9/\nXFJmrVLuu9dee01SlZWstdZawzYvwe1xzBO+Q7e7YrPCzuUhF4zd2muvLanaPzBo+kKSfvzjH0vK\nYS7+/f3RH/2RpMwunc03CTkKhhUIBDqDWLACgUBn0Np/ivEOmu00D2qIoQ6XqJSN9ccee6wk6eij\nj05lGFndSMv+pIMPPrhyj5TVxZJx350AgwK1yFUu2kCbXB3ib6gz0ctSpugYY51KowaV1ERU5Fmz\nZqWyL3zhC5JyO30vn0fXDwtXezD44m73ZGzcR7iKG5hRN1D1XcXDWO37BVFBUL1c7eC31OXOO+9M\nZU1VQnds8DdhDUSpSzmsoR6JL+VwlY022khSVfUhU4HPyy9+8YuScrs9ah4nBXPX+7LNfkkPqeGb\neeWVd5N5+rfA98eYer0322wzSdJqq60mSfr7v//7VIbx3Oce3wlrgZtheA/96E6TJg6xYFiBQKAz\nGJph9ft9eVJ/Vm1CGDxgEGPpvvvuK0m69NJLUxkr9ZlnnikpG2ulvKsdd7AkXXjhhZKyy/XEE09M\nZbfeequkLKVL2QeawKUrRn7YjQe9EfRIH3gQ5F133SUpSzlvJ65svx/JTtCh7/Dn+bAel4ruBGgD\nJD5S0iUpBmLa4vsFkcpI2euvvz6VsQ8Uhuh1Z5+dh6LgRsco7izPg2wHxezZsyvMnzlKHXy+EOQM\nU/FwG/rmiiuuqNRPyjmgML5L0s033ywpZxjx8ACCjQm89gBdd1wNg16vV8xiQniFG90JDIbNOvul\nPwhfcSZEH7hThRAjxsnnOGsALNaZdCkF+kQIhhUIBDqDRjYsl0jYINDPt9pqqzH3EyToLl9sXuj+\nhxxySCpjFb7jjjvSNUL8We333nvvVAbzYKe8u8GbhDXArDzUAkYFk/HAOe4nJIE2SZld0C8wSyn3\nAQGS/h7KYGhSZjRIardvNcnEWd/NL+VxhBV7KAJ2Lex5zkAJuSB8w/OiMS5eX+xSMDKvP+7xjTfe\nWFKVmbj7fxh4XXku7XaXPmC+/emf/mm6BrtjvnkQMPMBG56UQyToG88+gi2TrTFuX2q6/ajf71fa\nApIjvVcAABljSURBVIPkm3FGyDdG6BCai5QzSfzxH/+xpKo2wd+lLVvMY3LdSTmshzp4frEmtrpg\nWIFAoDOIBSsQCHQGQ6uEvV6vYuCF1pGA7corr0xlqEPQTz+Xba+99pKUVQhXlTDG+f0HHHCApKw6\nHH744akMVQk38t/93d+lMqfhg6K0l4s2Y7z1+pKYEFrN3iopGzdRP1APpLwf0ak66iQOCjdGo1LR\nnx693cSASV+6owTKTiS6q1Koh7jkvZ94FmqI9w/GX4/gxuHA7gc37tMvHs4APP3uMCjNWfrP5yxz\niDADnAJS3tuKyuZtpL/cVLDbbrtJyqqkq8QYoZlP7shq4u7v9/tj5gDqKW13tY959Yd/+IeSpDPO\nOCOVXXPNNZKymcbDMfhu/bDh73znO5Ky0d1DNGgf4QzetjiEIhAIfKDRyOjuKzkGbnadu4EUaYtU\nLJ2Uu99++0mqSljYmmdkYDWGdf3whz9MZew54xluLG/iIh7vSC4cBx5SwD4pHBBebwzIuJsJCfDf\n+b5IDLns1/PgVfZsYdAmsE9qJq0YDw+qRCrDupyZYKzGYOxZKc4991xJ2U3urHG8I+Bgbbj+pSyV\nqYMzbc/DNAxcshOQfMkll0iqhiLg5CBAFYeRlA3I5IviX68roR9SZlGEc/zVX/1VKoOF0DfOpJtk\npJDenQM+9xkv9pm6Awp2SUjNYYcdlspwbMDkPRwCDYHwDylrBbAvnzMEDZOBxevQJCtFMKxAINAZ\nxIIVCAQ6g9Z7CaG9xOj4WW3nn3++pKwSelpj0syiCrihDoOsx3ShGmHQdlUJlQZV0mNEPLJ2WLhK\nSFwUaoEnY4NCs4fRjeHQclQSoo6lnG7EU9dioEW1dgqNgRoq7VHfbU5EdnWJv4m38Vgj0o2gevue\nSVRBxtqdC7SBNCtSNly7IRigbqNiu+rZdDx9TJg7qISuxhFjRV1dVUVd5pTyb3/726kMNchVfpwT\nqLQeI7XttttKyjFMPgfawNUs1E2cGK76YiphDrqZB2cCO0i8bph+cKhJOe6KyH534vDNM4ZE+EvV\nfbWDIhhWIBDoDFozLNyxGIBJVyzlxPNIJk/aT5YGVnN3YZdOcMZQyv0u+TFowwDc8NhkFQculWFx\nGIm9DHd96bRmQh44EdgZKPd5pDkMgmd6xDhMBRbkSfuaMKyScwGWSv/6XkbawL4537eG44AxcGcH\n+wxvuummdA0XOOPprJhxh+W5Ab/pAQ3OIMhsgbOG9ki5/2F5fvYj98GifGz22WcfSVUHBoZ06g8z\nk/JBK+wC8DnTNNK9Doz3MNWDDjooldEG5pLvqKifWO1GdJizs0UM6kTN+3cLG+fbdwdTEwTDCgQC\nnUFrhoXURYIhtaSc0B+blIcu4BIm4M6zKnAAwz333JOuYQfATuRBeOzDQhL4e/yk5WFRCuCDvbku\njt0Ae4AzJupN/3jYBzYol+IEbiLh6V8pt2tUx0BRb7d7IAH511kEbcde5VKW8A2YsPc7rMXtQbA6\nJLb3Sz3flktlz9wwDJxF8gze6f3PfORgCrcTYo+EAXkZTNGZHCmSYcJ+8jnvpMyZmdtnh4W3k4BP\nGI+HlRCwjX3L97NyP3XykBlsuX7yM/2JluX1Z87Uj1ZrimBYgUCgM4gFKxAIdAatT81x975UDV1A\nhUFVcjpIWmCotKtYGPGc/nMN6uplqF1QVzduOg0eFt5O1EPe4a5wKDTt9HfWU8O6ARPq7JQbYzX3\n+bN4RimJXZNId+Cqr0dc19+F8ZboaKf+uLIJb/H64Lr3kASMzBh/UTWkbPSl//13TVV8V3vZ34m6\n7SlPSFvEOz31NA4J4KlkcBDQR1LerYBh3VVi1C36yw3tTUI3er2epkyZUlEJeTZzyU/tYc8jc9e/\nY5wEnDPoiRrZaeC7D/gWeL4711C7+V7dudAkuWYwrEAg0Bm0tt4iWZFgfkgABniM4r73jcA5JIIb\n65HgLu2RrBgSnZGxUhPC4NJ9VIGjSAbc76VARCQwkk3Krn9YjDMW6u0Mh2sYK92BMGrQvlJ/weY8\nWwFjXHfXS2NTSDsTgjV6qAO/pZ2l068JTvRxaJryupQ2m/p41gUYBGEd/j7CbRhTfyYM0RkxjpN6\nOIqUHTPOekp1HRZeXwzdzFVnkowz89HTdNNO6ujP5JrPGdga/eksEwbn4TwgsjUEAoEPNBoxLJd4\nuLaRHs58CEFAf3VWgju35PJGwrq0qjMP38pTdwO3OSbpvcD7kRSlk6dhCR5QSRnMw93esBi/n3Jn\nHACJ1MZeVUIp6LT0LqQlktr7mQBi+slZQil4F4bFv4MeHNKGfdTBmHjoCMGhtMfrDrtg3Lz9MBVn\n9LSNMfX0zqWQklHAx6vej57qmPqWskWgEcGSPeU3bfH5zzthkm1Pdx4PwbACgUBnEAtWIBDoDFob\n3etJ4Eou/VICPwB9dFWJZ5TUovGM6G0yFkwE2kIb3IAM7eXf0n4p7vc6ogqWjPujptLDop5EzzFI\naEEpffJ46k+b8yObgjnnThLqQQiO17k+B9zEUVKh6+31MZ2T7aWdPm7MM0wWXlccARjPvQzTT2m3\nBfeNp5a2RTCsQCDQGfSGYSW9Xu8dSa9MeOPciyX7/f4i493wAWij9OFo54RtlKKdHcJg7ZxMNSoQ\nCARGiVAJA4FAZxALViAQ6AxiwQoEAp1BLFiBQKAziAUrEAh0BrFgBQKBziAWrEAg0BkMtTVngQUW\n6HtOnYlAqP7cEOu10EILaebMmT+fKDht6tSp/TaHALzfWHTRRT8U7XzjjTcmbKMkzTPPPP1RHZs1\npzFt2rSBxlKS5ptvvr7noeoSBp2z0pAL1iKLLKLvfve7A98/Ny1Ye+65p3q93oSRwNOnT9dee+3V\n6B2lPXP1/WSlvijtu2uK448/fuB2cpZeU3hdh02TUm/nsL8/7rjjBorqXmCBBbT//vsP9WypnCan\nnhJmsuf1GWecMdBYSu8K5KOPPrrxu8Zry0Rj03Ysv/rVrw7cztGcFzUgBp3g45WxsXIycl4Ng9KG\nUDaX+qZtNomWNsfyUXhb2CDL872sngfr/RIEvLdU71KuLtrpG2HZxE7/eE6wUeeIGga8m7r6huWm\nH+bcILDfC7TBWWj97ACvf0ko15m6z//Y/BwIBD60iAUrEAh0BiNTCaGKTgehlKgJngYZlQBVwA9b\nQNVwqkmie+iqH79UV7cmg4LX8375YRLkv6LMU9FSFw41IPWulNvpucDIT1RKxVx/N8+cDEDlvZ2k\nCKbe7oChbtTb8yXRvlIaZMbf86iRWniyUkKD8dIUk8PNj2WjjuQIczWKdriazPyl/T7O3Feqw6jn\n73jfJt+hn+JN3ZjHXm9yZXm/oDbXD7bw+0alGgbDCgQCnUFrhoX0Q8JwSISUpS1MwI/6qR8hVPLG\ncEySlA9HLWXkRAJw2EUpU2lbUCcM655xFKM7UtkZAYn5+V1JYpeOsaJ9fggCB5LSTo7BksoZIJuA\ndiIlnS1SXxiQH4xbz0y68sorp79hYk8++WS6Rn9wpJQfdwWLLrGcycjE6gyCsXz55ZclVZn/sssu\nW6mfH+ZLXzz++OPp2kYbbSQpH6RKW6V8IAXHgzkzGzVzrh/aKuVsopT5wcCMJePsdUPTccbEvIBh\n+XdO+xjvUjbeYRAMKxAIdAYjy+mO1PHDNZGs3OPsq+7OdtsOx5ffd9996Ror9VprrSWpynCwJZRy\njY9KIsOCeJ674TkEdvr06ZKyFJKyVIZpIbmlzCBd8nGMOn3FwZ7SWPsdLFWqHq/WBkhOWKK/HylM\nP3vfbr/99pLyWHBEuZSZIUe7ed2RxvSPlMeWslGHOdRzj8PkpDwP6Ws/pp22MRf92C5sQG6/Yaw5\nst6fxX0c6+7fjR+4OgrAfJifUm4fY+qsbumll67cw9FnUma7Pt84Buz111+XJG2yySapDPbFHPDg\n1iaaUDCsQCDQGcSCFQgEOoPWKiHqASqeU1sMkag8frrs+uuvLymrN26QhYKussoq6RrUElXCVQhU\nB1RJp5pO99sAmo5B0mnypptuKimreG+//faYevM77vW6PfTQQ+ka6gqn766++uqpDKp+3XXXSaqq\na66GDgs3+qMmbbnllpKqoQvXXHONJOmJJ56QVB2fVVddVVIeg8svvzyVoQ7ceeed6dpWW20lKTsO\nXL3CAM+88aPdRmGQrquYroIxXo8++qgk6YEHHkhlqOs777yzpKqJg7ntajJq8bPPPispt1nKcxbn\nhKv3bUIASseS0X/uEAA4HLxfr776aklZBfbwoh/+8IeSpI033jhd+9znPicpm0bcUYHKy3tcJXQn\nxKAIhhUIBDqDRgzLJbKHHkhViYxLnhXXDbG4yzHYLb/88qmMld3dqS+88IKkLKXcwP7iiy9KygZ5\nZ2ujQn3PlTM3DPCwTO8DpNy9994rSXrqqafGPHPatGnpGqzlpptukiRdfPHFqQy2teGGG0rKzEUa\nnVSmz5H4GF6l3Hbc9c6mka6PPPKIpKr0pE3rrLNOuvbqq69KymzUjbjMjVLAaNOgSm8j48T8dFc7\n9brnnnskVcdyxRVXlJTn/LnnnpvKTjnlFElVtgbTh33dfffdqYw5S1udeTQNlO31esX3ow3cdddd\nqWzttdeWJB155JGSMhOSMuOH6fo4z5gxQ1JVU9huu+0kZQ3AnWXPP/+8pOyU8DCZJvuBg2EFAoHO\noBHD8lUc9ywS1VkU7mlWeOwiUtaZYSceHIiE8ZUdlzDX3ObBb7EfuM7dJqDSpXk9kBLGJ2XbBhLG\nXdtILtiQB1vi7kbaSdVgUKlqq0PiwRZmzZqVytzOMyycfVA/bFHuokZi0xbaK2XWAcPysWP8XSrf\ndtttkjK78W1bSPjNN99cUjXswxnfMPA2wo5hWG6LW2GFFSRlG5azHepx2WWXSaoyRjQEbI/+W8bt\n6aefTmUwbWyPpWwJTeDznbmDFuPPRRthfnq96YPzzz9fUjkcwucsGg0smyBv/xs2VbKXDoNgWIFA\noDOIBSsQCHQGQ+lLvV5PvV6vEjaAsbBkKIV64/Z0OkiE7XLLLSepSvt95z7AhXzjjTeOeQ+qIAbT\nUcFd1FDh0q581FuoP25sKRvD11xzTUlV4zJ02etNuzCwo2JJ2RCOodqj/dtEg7vBnvAL1AAMzVJW\nCVEnLrzwwlR21VVXVeqIUVmS7rjjDklVg+tZZ50lKe/HO/TQQ1MZZgXULO8fj5YfBj6WqE2odB4e\ngmrG/e5UQhUmFMH3IBIh7/voiHSnD/fdd99URl/yjbhKSCR9W9SdQIRjSNn0gEro+yLpf8bZjeg7\n7rijpGoYxj/8wz9IkpZZZpnK76WsauLE2G+//cbUYRgEwwoEAp3B0BbpOsPCaAgzcKlTlxQuyWAh\nSHc3FnKfGyl5D5LP34ORHdbjbuo2BkyXOkhXwik8uwAGaiQZkkbKjIw+c0N5KfMDxur7779fUjVP\nEcwKaeyG7TY5lHw86ddSimCyDtAmWJUk3XrrrZKU8sR7kCQsERe3P/+www4bUwfGn2ci6aVmrvB6\n3zBOMDgf55VWWkmSdOqpp1bqLuVgWNiys0+cQM4ueC/sy8eeUAfmirfftY1hMHv27IpDjHoyzzwI\n9vrrr5eU59A222yTytj/+rWvfU1SDsmR8nf64IMPpmuwLoz8rmHgkEJjcAbaxFEUDCsQCHQGQzGs\nfr+v2bNnV1zQ/I0dhxVVyqsxIQAuRbDlYMtyJoRe79sF/LlSdcsKTK6k+7ODvAnctuNsRspbi6TM\nHAgJcLsHjADd3QMEkeZuC6I/CCb1IFiei+R297GHUgwLZ3iwU2xZsEcpS97vf//7Y55x8sknV+p7\n5ZVXpjLCBpzpfOMb35AkHXTQQWPeA2PG9uWspUk+pV6vVxlLwmDoTw9C5t20w/tm2223lSQdc8wx\nkqQbbrghlZ1++umSqqyBsWQrk7NDQioIBXDbzjBH6Tl6vV7lm4EtE0LiTAk2BLPy0IV64PYGG2yQ\nyti242DsYJC+NYffsgXPv1EPFh8UwbACgUBnEAtWIBDoDBqFgXvyOtQ8VCY3qhExC2UkJEFSOtwS\nKur74qDjHk1LJP1qq60mqWrYQ/WEXru7tM0BBqXUuRiVvb7s9yN62XfxQ/Vpu7uvMVpjXJak3//9\n35eU+9OdEUSMo964it2knaVQCJ6NKlRyLlxyySWSqvOAccEY66oJz/JwFUwC9WhwKfcfxnB3ofs7\nB8WUKVMqKjOhEYyp7z5ADWLfnWcYwSzB2M+cOTOV7brrrpKy+iVlozwmDldLb7/9dknZ6O47J1wN\nHRTsI3RzBH/vueeekqpOD5w6qNi+C4VwDOaqG+tps+9/RRWk3ltvvXUqo99xmnmiTlcdB0UwrEAg\n0BkMzbD6/X5FMsNCMDC6sZVVm8wDnveJXd8wMg95wHDpkg8mRvAhbErKwaes+s422gRUupEYqcM7\nXOrghsaAzD40L4NRLL744qmMsA12u0tZKtJ2DPPS2B3vF110USprE77hrnAYDO1EEku5zTA7d4TA\nqAhn8L7D8eF79nbYYQdJWTq7MXePPfaQlPePuqPCA1IHxezZsytMFeaG9CeXl5T7HQO7s18YCuNG\nG6Q8z9zozjtpv4eIsK+S0APXCpwJDYp+v69f//rXFUZYZ8u+L5VvhmBSD1Pg++Ea90r5m/YAXgJp\ncUK4o4vx4h53lrmWNCiCYQUCgc4gFqxAINAZNDK6O7WFgmJYdBWmngzO42kAcRmuQvBMT3mBqgLl\nhrJLOcqY+BU3iI/q1Bzo9AUXXCCpGlNDChZUNm8LRk3aVFLdHn744TH3Y5z08+JoHzE8HofVJNKd\n37hKiDqAaurqGGrPTjvtJKmcoplo+JtvvjmVkfDPo/ZRl4iJ4tQdKUf0Y4h3Q3vTE2VKqYPpa+9j\nHD6Mkye2Y84x1139QnX2BI3UFTOAn5iEOkkbPQ7N91wOCzdk02ZUusMPPzyVHXfccZKkM844Q1J1\ndwZqG+qcq9Mkj/S4RPqIMfS0O3zD9LWnD4/0MoFA4AONRnsJPTIYgx4uUGcLSFZWV9+XReTrEUcc\nIam6YuMadqMukohIZH4vZcM9TM6laRuG5W5oGASsxMM3kBoY230nAP0CU/H9ZEg1byfXqLcnNqTf\nkdzOsNo4F5y11E/79ahr7rvlllskVfd6Inlhhv472kA4hJQj6elHDzugz2DTboRumpDR+weG+txz\nz1X+L2VDM0ZikixK+eAFGJAf/IFTxQ3UOIpgiB6lT5+UzphswyI9FId5SZiIR+bD9nCSEPrgv8Mg\n78yYcfL+ZA1g7rrjDeZIv3jIRqRIDgQCH2g0ElfOsHBN4gZ3psQqjsvbwxQoI2DOJTKSyZP28070\nXt/Rji0BSTmKhP7vBerhgZFIIo7BcuYDe4JRuK0Au1jJ1Yv0dulG/2EX8IMeeH5b0C76zYNgGVvu\nccZMO2mf2xFhDL5fjfHH1uUud2xj2Dw9QLZ0uvewgLnRjx7MSztIK+xMGkaG3dXZQj2PmJQDR0tp\nwJn32MqcVTUJHAXOWmBvpYwoX/ziFyXl/aEcHSdlVgkr8jAFbGTOrukz7H8eekLYBnPGmXQTu2sw\nrEAg0BnEghUIBDqDRiph6Rw7VAhXeaDQUGLfG4gBkvSrrkahHrhRk2tQaKfqqC3UwWlnm1NzvJ28\nF3WidDIOdXKqiwGTfnGVmb11rg6hWqMW+Xug+DggXC1tcy6hg/rxrxuDMTZjBHdnAdQfVd3rRvtK\n+yIJCfFzHkmJTJ+7Wt/0dCAfE/6mz3wfH04D5qqbHuh/1CiPTkfFdaM3hnj6zdVZxpm5O6p0164S\n8t3RztJ8LplMUOdpkyc45Jv0ECXGkOh3dzww5vSdh100SRUUDCsQCHQGrRkWgEk4W8CQTpIwNzpy\nH1LHAyq55sZQ2Av3+apfDwT0+rVJHezgHUgMD2ZEOlJHP2iCPqC9bsAssRgM67A2D8tAeiIxm7q/\nxwMSn3c5o6ENu+++u6TqmHEfxudS+uZSACh95eyL0BUPMgRNw1R8HtA23uNhKAQhY+h3Zwbziro6\nK4JZOcMicR+swsv4LX04qlAcB+y8FGgMc2RM/IAK7qfMz59kfnhwK4G3zEvPyIAW4W0HpWsTIRhW\nIBDoDJobeP4fMA+kVilNMalQ3T2NhKnbE6SsO7udCvZU0rlLOvpkgTa4FCT0gGteb/qFLRsu6dmy\n4cwTVgkTcyZZ6qtRwPsNycs1L0O6YkNzCYlULo0BjMRZFDauUpriUbevDuYQrNDHi7AEWFHpQAjq\n59oELM1ZLyy5FCBJP42KTZXA3GG8fCsRLIi6OVuEeTJePh5oDF5vwmtK4RO00+dxGwTDCgQCnUEs\nWIFAoDNorRKipmDgK2UjKBm+oZ2lslLUNvc12X80SlAPV31QA0rG5bqK5HQZo7KryvX7PaxhTqi8\n9XeUDKNeXzDejoKSise1uiNhToA2lswM9VODxsuI4eaPUurm0lx5P+F9jEpYOhuzXl/PmoJq533G\n/Gde+HtG3fZgWIFAoDPoDeP27/V670h6ZcIb514s2e/3xz307QPQRunD0c4J2yhFOzuEwdo5qjil\nQCAQmGyEShgIBDqDWLACgUBnEAtWIBDoDGLBCgQCnUEsWIFAoDOIBSsQCHQGsWAFAoHOIBasQCDQ\nGcSCFQgEOoP/A2aQjR27KQAFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x8e2e400>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hidden_layer(final_theta)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
